package com.chance.cc.crawler.development.bootstrap.pcauto;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.Map;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.turnPageItem;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Site_Info;

/**
 * @Author: ZhaoHhuan on 2021/9/18 16:23
 * @Email: 18638575967@163.com
 * @Description:
 *      将文章地址放入redis队列中
**/
public class PcautoArticleUrlListToRedisStart {
    public static final String domainId = "pcauto";
    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {
       // testArticle();
        //testVideo();
        //video();
        createItemRecordToRedis();
    }

    //将文件中的地址生成record放入redis中
    private static void createItemRecordToRedis(){
        String site = "article";
        String siteBiz = "simple";
        String siteInfo = "simple-20210930";
        String path = "E:\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\pcauto\\pcauto_json.txt";

        RedisReader redisReader = new RedisReader("192.168.1.215",6379,4);
        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8"));
            String readLine = null;
            while ((readLine = bufferedReader.readLine()) != null) {
                if(StringUtils.isBlank(readLine)){
                    continue;
                }
                JSONObject jsonObject = JSONObject.parseObject(readLine);
                String url = jsonObject.getString("article_url");
                System.out.println(url);
                CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                        .startPageRequest(domainId, turnPageItem)
                        .domain(domainId)
                        .httpUrl(url)
                        .releaseTime(System.currentTimeMillis())
                        .httpConfig(HttpConfig.me(domainId))
                        .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                        .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*62,null))
                       // .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        //.resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                        .proxy(proxy)
                        .build();
                requestRecord.tagsCreator().bizTags().addDomain(domainId);
                requestRecord.tagsCreator().bizTags().addSite(site);
                requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
                requestRecord.tagsCreator().bizTags().addCustomKV(Tag_Site_Info,siteInfo);
                String requestKey = requestRecord.getRecordKey();
                //确保key值唯一
                String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+requestKey);
                redisReader.mapPush("crawler-pcauto-article-simple-queue_map",shaKey, JSON.toJSONString(requestRecord));
                redisReader.listPush("crawler-pcauto-article-simple-queue_list",shaKey);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    //将文件中的地址生成record放入redis中
    private static void video(){
        String site = "video";
        String siteBiz = "news";
        String siteInfo = "bmw-20210902";
        String path = "F:\\需求文件\\19_taipingyang_news.json";

        RedisReader redisReader = new RedisReader("192.168.1.215",6379,4);
        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8"));
            String readLine = null;
            while ((readLine = bufferedReader.readLine()) != null) {
                if(StringUtils.isBlank(readLine)){
                    continue;
                }
                JSONObject jsonObject = JSONObject.parseObject(readLine);
                String url = jsonObject.getString("article_url");
                CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                        .startPageRequest(domainId, turnPageItem)
                        .domain(domainId)
                        .httpUrl(url)
                        .releaseTime(System.currentTimeMillis())
                        .httpConfig(HttpConfig.me(domainId))
                        .filter(CrawlerEnum.CrawlerRecordFilter.key)
                        .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domainId))
                        .addFilterInfo(FilterUtils.dateRangeFilterInfo(0,new long[]{1629561600000L,System.currentTimeMillis()}))
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                        .proxy(proxy)
                        .build();
                requestRecord.tagsCreator().bizTags().addDomain(domainId);
                requestRecord.tagsCreator().bizTags().addSite(site);
                requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
                requestRecord.tagsCreator().bizTags().addCustomKV(Tag_Site_Info,siteInfo);
                Map<String,Object> extras = new HashMap<>();
                extras.put("listUrl","html");
                extras.put("articleUrl",url );
                requestRecord.getHttpRequest().setExtras(extras);

                String requestKey = requestRecord.getRecordKey();
                //确保key值唯一
                String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+requestKey);
                redisReader.mapPush("crawler-pcauto-video-simple-queue_map",shaKey, JSON.toJSONString(requestRecord));
                redisReader.listPush("crawler-pcauto-video-simple-queue_list",shaKey);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    public static void testArticle(){
        String domain = "pcauto";
        String site = "article";
        //https://www.pcauto.com.cn/news/2648/26489891.html
        //https://www.pcauto.com.cn/news/2651/26519431.html
        //https://www.pcauto.com.cn/news/2626/26266211.html
        //https://www.pcauto.com.cn/pingce/2643/26438291.html
        String url = "https://www.pcauto.com.cn/pingce/2643/26438291.html";
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .recordKey(url)
                .httpUrl(url)
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .proxy(proxy)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*62,null))
                .build();
        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);

        DevCrawlerController.builder()
                .triggerInfo(domain,domain + "_forum_article_trigger",System.currentTimeMillis(),domain+ "_forum_article_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
                .requestRecord(requestRecord)
                .build("com.chance.cc.crawler.development.scripts.pcauto")
                .start();
    }
    public static void testVideo(){
        String domain = "pcauto";
        String site = "video";
        String url = "https://pcauto.pcvideo.com.cn/video-65709.html";
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .recordKey(url)
                .httpUrl(url)
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*100,null))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .needWashed(true)
                .proxy(proxy)
                .build();
        requestRecord.tagsCreator().bizTags().addSite(site);
        Map<String,Object> extras = new HashMap<>();
        extras.put("listUrl","html" );
        extras.put("articleUrl",url );
        requestRecord.getHttpRequest().setExtras(extras);
        DevCrawlerController.builder()
                .triggerInfo(domain,domain + "_forum_article_trigger",System.currentTimeMillis(),domain+ "_forum_article_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
                .requestRecord(requestRecord)
                .build("com.chance.cc.crawler.development.scripts.pcauto")
                .start();
    }

}
